This notebook includes code for retrieving and parsing datasetsfrom the bulk download facility maintained by the U.S. Energy Information Administration. Datasets currently retrieved include:

if(!("jsonlite" %in% installed.packages())) install.packages("jsonlite")
library(here)
library(readr)
library(tidyverse)
library(stringr)

t(manifest$dataset[[2]]) 
                   1                                                                                                                                                                                                                                           
last_updated       "2016-01-04T10:35:59-05:00"                                                                                                                                                                                                                 
category_id        "1370522"                                                                                                                                                                                                                                   
name               "Annual Energy Outlook 2015"                                                                                                                                                                                                                
data_set           "AEO.2015"                                                                                                                                                                                                                                  
identifier         "AEO.2015"                                                                                                                                                                                                                                  
title              "Annual Energy Outlook 2015"                                                                                                                                                                                                                
description        "The Annual Energy Outlook (AEO) from EIA.gov provides long term forecasts (25 years) of U.S. energy production, consumption, and trade for the United Stated of electricity, petroleum, natural gas, coal, nuclear, and renewable sources."
keyword            "Annual Energy Outlook AEO EIA Energy Information Administration long term forecast projection united states production consumption trade electricity petroleum natural gas coal nuclear renewable hydroelectric wind solar"                
publisher          "U.S. Energy Information Administration"                                                                                                                                                                                                    
person             "Mark Elbert"                                                                                                                                                                                                                               
mbox               "data@eia.gov"                                                                                                                                                                                                                              
accessLevel        "public"                                                                                                                                                                                                                                    
accessLevelComment "series copyright field contains further information "                                                                                                                                                                                      
accessURL          "http://api.eia.gov/bulk/AEO2015.zip"                                                                                                                                                                                                       
webService         "http://api.eia.gov"                                                                                                                                                                                                                        
format             "XML and JSON data API, JSON download file"                                                                                                                                                                                                 
temporal           "annual"                                                                                                                                                                                                                                    
modified           "2016-01-04T10:35:59-05:00"                                                                                                                                                                                                                 
HEAD_response 
Response [http://api.eia.gov/bulk/SEDS.zip]
  Date: 2020-03-16 23:57
  Status: 200
  Content-Type: application/zip
<EMPTY BODY>
# setwd("C:/Users/Christina Chung/Desktop/EO43")
# fuel <- read.csv('fuel_uncleaned.csv')
# library(dplyr)
# library(stringr) # for replacing strings
# fuel <- as.data.frame(t(fuel))
# fuel <- fuel[,2:10]
# names(fuel)<-lapply(fuel[1,],as.character)
# fuel <- fuel[-1,]
# colnames(fuel)[1] <- 'Year'
# fuel[,1] <-str_replace_all(fuel[,1],'Year','')
# colnames(fuel)<-str_replace_all(colnames(fuel),' ','_')
# for (i in c(3,6,9)){
#   fuel[,i]<-as.numeric(gsub(",", "", fuel[,i]))
#   
# }
# fuel_ordered <-fuel[nrow(fuel):1,]
# write.csv(fuel_ordered,file = 'fuel.csv')
get_EIA_bulk_json <- function(url){
  
  require(utils)
  require(here)
  require(stringr)
  require(zip)
  require(fs)
  
  os <- Sys.info()[['sysname']] 
  sep <- ifelse(os == "Windows", "\\", "/")
  
  zip_filename <- word(url, start=-1, end=-1, sep=sep)
  
  download_dir <- here("data", "raw_data", "EIA")
  path2zip     <- paste(download_dir, zip_filename, sep=sep)
  
  download.file(url = url, destfile = path2zip, method = "libcurl")
  
  unzip(path2zip, exdir = download_dir)
  
  unzipped_filename <- zip_list(path2zip)$filename
  path2unzip <- paste(download_dir,unzipped_filename,sep=sep)
  json_filename <- str_replace(unzipped_filename, ".txt", ".json")
  path2json <- paste(download_dir,json_filename,sep=sep)
  
  file_move(path2unzip, path2json)
  
  file_delete(path2zip) 
  
  return(path2json)
}
SEDS_va_elec
Error: object 'SEDS_va_elec' not found
url <- "http://api.eia.gov/bulk/EBA.zip"

path2json <- get_EIA_bulk_json(url)
trying URL 'http://api.eia.gov/bulk/EBA.zip'
Content type 'application/zip' length 284939963 bytes (271.7 MB)
==================================================
downloaded 271.7 MB
EBA <- stream_in(file(path2json))
opening file input connection.

 Found 500 records...
 Found 1000 records...
 Found 1500 records...
 Found 2000 records...
 Found 2500 records...
 Found 2789 records...
 Imported 2789 records. Simplifying...


closing file input connection.
(EBA %>%
  as_tibble %>%
  filter(geography == "USA-VA") %>%
  flatten() %>%
  mutate_all(as.character) -> EBA_va)
Error in eval(lhs, parent, parent) : object 'EBA' not found
(ELEC_va <- read_fst(path = here("data", "ELEC_va.fst")))
LS0tCnRpdGxlOiAgICAiVmlyZ2luaWEgQ2xlYW4gRW5lcmd5IFByb2plY3QiCnN1YnRpdGxlOiAiRXh0cmFjdCwgdHJhbnNmb3JtIGFuZCBsb2FkIGRhdGFzZXRzIGZyb20gVS5TLiBFbmVyZ3kgSW5mb3JtYXRpb24gQWRtaW5pc3RyYXRpb24iCmF1dGhvcjogICBBcnRodXIgU21hbGwKZGF0ZTogICAgICJWZXJzaW9uIG9mIGByIFN5cy5EYXRlKClgIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpUaGlzIG5vdGVib29rIGluY2x1ZGVzIGNvZGUgZm9yIHJldHJpZXZpbmcgYW5kIHBhcnNpbmcgZGF0YXNldHNmcm9tIHRoZSBbYnVsayBkb3dubG9hZCBmYWNpbGl0eV0oaHR0cHM6Ly93d3cuZWlhLmdvdi9vcGVuZGF0YS9idWxrZmlsZXMucGhwKSBtYWludGFpbmVkIGJ5IHRoZSBbVS5TLiBFbmVyZ3kgSW5mb3JtYXRpb24gQWRtaW5pc3RyYXRpb25dKGh0dHBzOi8vd3d3LmVpYS5nb3YpLiBEYXRhc2V0cyBjdXJyZW50bHkgcmV0cmlldmVkIGluY2x1ZGU6CgogICogW10KCgoKCmBgYHtyIHJlYWQgaW4gcGFja2FnZXN9CmlmKCEoImpzb25saXRlIiAlaW4lIGluc3RhbGxlZC5wYWNrYWdlcygpKSkgaW5zdGFsbC5wYWNrYWdlcygianNvbmxpdGUiKQpsaWJyYXJ5KGhlcmUpCmxpYnJhcnkocmVhZHIpCmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KHN0cmluZ3IpCmBgYAoKCiMKCmBgYHtyIHJldHJpZXZlIHVwZGF0ZWQgbWFuaWZlc3QgZmlsZX0KdXJsIDwtICJodHRwOi8vYXBpLmVpYS5nb3YvYnVsay9tYW5pZmVzdC50eHQiCgptYW5pZmVzdCA8LSBzdHJlYW1faW4oZmlsZSh1cmwpKQoKIyBzdHIobWFuaWZlc3QpCgojIGZsYXR0ZW4obWFuaWZlc3QpCgp0KG1hbmlmZXN0JGRhdGFzZXRbWzJdXSkgCgpgYGAKCgoKYGBge3IgZGVmaW5lIGZ1bmN0aW9uIHRvIGZldGNoIGEgc2luZ2xlIHppcHBlZCBmaWxlIGZyb20gYSBzcGVjaWZpZWQgVVJMfQojIGdldF9yZW1vdGVfemlwX2ZpbGUgPC0gZnVuY3Rpb24odXJsKXsKIyAgIHJlcXVpcmUoaHR0cikKIyAgIHJlcXVpcmUoUkN1cmwpCiMgICByZXF1aXJlKHJlYWRyKQojICAgcmVxdWlyZShzdHJpbmdyKQojICAgcmVxdWlyZSh6aXApCiMgCiMgICBwcmludCgiU2VuZGluZyBIRUFEIHJlcXVlc3QuLi4iKQojICAgSEVBRF9yZXNwb25zZSA8LSBIRUFEKHVybCwgdGltZW91dCgzMCkpCiMgICBwcmludChwYXN0ZSgiSEVBRCByZXNwb25zZSBzdGF0dXMgaXM6IiwgSEVBRF9yZXNwb25zZSRhbGxfaGVhZGVyc1tbMV1dJHN0YXR1cykpCiMgICAKIyAgIGlmKEhFQURfcmVzcG9uc2UkYWxsX2hlYWRlcnNbWzFdXSRzdGF0dXMgPT0gMjAwTCl7CiMgICAgIAojICAgICAgIHppcF9maWxlbmFtZSA8LSB3b3JkKEhFQURfcmVzcG9uc2UkaGVhZGVycyRgY29udGVudC1kaXNwb3NpdGlvbmAsIDIsIHNlcD0iPSIpCiMgICAgICAgcGF0aDJ6aXAgPC0gaGVyZSgiUmF3X0RhdGEiLCAiRUlBIiwgemlwX2ZpbGVuYW1lKQojICAgICAKIyAgICAgICBwcmludCgiU2VuZGluZyBHRVQgcmVxdWVzdC4uLiIpCiMgICAgICAgR0VUX3Jlc3BvbnNlIDwtIFJFVFJZKCJHRVQiLCB1cmwgPSB1cmwsIHRpbWVzID0gMywgd3JpdGVfZGlzayhwYXRoMnppcCwgb3ZlcndyaXRlID0gVFJVRSksIHByb2dyZXNzKCksIHZlcmJvc2UoKSwgb3ZlcndyaXRlPVRSVUUpCiMgICAgIAojICAgICAgIEdFVF9yZXNwb25zZV9lcnJvciA8LSBodHRwX2Vycm9yKEdFVF9yZXNwb25zZSkKIyAgICAgCiMgICAgICAgcHJpbnQocGFzdGUoIkdFVF9yZXNwb25zZV9lcnJvciBzdGF0dXMgaXM6IiwgR0VUX3Jlc3BvbnNlX2Vycm9yKSkKIyAKIyAgICAgICBHRVRfcmVzcG9uc2VfaGVhZGVycyAgIDwtIEdFVF9yZXNwb25zZSRhbGxfaGVhZGVycwojIAojICAgICAgICMgc2luayhmaWxlID0gc3RyX3JlcGxhY2UocGF0aDJ6aXAsIi56aXAiLCJfR0VUX3Jlc3BvbnNlX2hlYWRlcnMudHh0IiksIGFwcGVuZCA9IEZBTFNFKQojICAgICAgICMgcHJpbnQocGFzdGUoIkdFVF9yZXNwb25zZV9lcnJvciA9IiwgR0VUX3Jlc3BvbnNlX2Vycm9yKSkKIyAgICAgICAjIHByaW50KCJHRVRfcmVzcG9uc2VfaGVhZGVyczoiKQojICAgICAgICMgcHJpbnQoR0VUX3Jlc3BvbnNlX2hlYWRlcnMpCiMgICAgICAgIyBzaW5rKCkKIyAgICAgCiMgICAgICAgaWYoIShHRVRfcmVzcG9uc2VfZXJyb3IpKXsKIyAgICAgICAgIHVuemlwKHBhdGgyemlwLCBleGRpciA9IHdxcF9mb2xkZXIpCiMgICAgICAgICBwYXRoMmNzdiA8LSBzdHJfcmVwbGFjZShwYXRoMnppcCwiLnppcCIsIi5jc3YiKQojICAgICAgICAgd3FwX3NhbXBsZXMgPC0gcmVhZF9jc3YocGF0aDJjc3YsIGNvbF90eXBlcz1jb2xzKC5kZWZhdWx0ID0gY29sX2NoYXJhY3RlcigpKSkKIyAgICAgICAKIyAgICAgICAgIHJldHVybih3cXBfc2FtcGxlcykKIyAgICAgICAKIyAgICAgICB9IGVsc2UgcHJpbnQocGFzdGUoIkVycm9yIEdFVCdpbmcgZmlsZSBmb3IiLCB1cmwpKQojICAgICAKIyAgICAgfSBlbHNlIHByaW50KHBhc3RlKCJIRUFEIHJlcXVlc3QgZm9yIiwgdXJsLCJmYWlsZWQuIikpCiMgICAgIAojICAgfSBlbHNlIHsKIyAgICAgICBwcmludChwYXN0ZSgiRmlsZSBuYXJyb3dyZXN1bHQuemlwIGFscmVhZHkgZXhpc3RzIGluIGZvbGRlciIsIHN0YXRlX25hbWUpKQojICAgICAgIHppcF9maWxlbmFtZSA8LSAibmFycm93cmVzdWx0LnppcCIKIyAgICAgICBwYXRoMnppcCA8LSBoZXJlKCJSYXdfRGF0YSIsIldRUCIsIHN0YXRlX25hbWUsIHppcF9maWxlbmFtZSkKIyAgICAgICB1bnppcChwYXRoMnppcCwgZXhkaXIgPSB3cXBfZm9sZGVyKQojICAgICAgIHBhdGgyY3N2IDwtIHN0cl9yZXBsYWNlKHBhdGgyemlwLCIuemlwIiwiLmNzdiIpCiMgICAgICAgd3FwX3NhbXBsZXMgPC0gcmVhZF9jc3YocGF0aDJjc3YsIGNvbF90eXBlcz1jb2xzKC5kZWZhdWx0ID0gY29sX2NoYXJhY3RlcigpKSkKIyAgICAgICAKIyAgICAgICByZXR1cm4od3FwX3NhbXBsZXMpCiMgICB9CiMgIAojIH0KYGBgCgoKYGBge3Igb2xkIGNvZGUsIGV2YWw9RkFMU0V9CiMgc2V0d2QoIkM6L1VzZXJzL0NocmlzdGluYSBDaHVuZy9EZXNrdG9wL0VPNDMiKQojIGZ1ZWwgPC0gcmVhZC5jc3YoJ2Z1ZWxfdW5jbGVhbmVkLmNzdicpCiMgbGlicmFyeShkcGx5cikKIyBsaWJyYXJ5KHN0cmluZ3IpICMgZm9yIHJlcGxhY2luZyBzdHJpbmdzCiMgZnVlbCA8LSBhcy5kYXRhLmZyYW1lKHQoZnVlbCkpCiMgZnVlbCA8LSBmdWVsWywyOjEwXQojIG5hbWVzKGZ1ZWwpPC1sYXBwbHkoZnVlbFsxLF0sYXMuY2hhcmFjdGVyKQojIGZ1ZWwgPC0gZnVlbFstMSxdCiMgY29sbmFtZXMoZnVlbClbMV0gPC0gJ1llYXInCiMgZnVlbFssMV0gPC1zdHJfcmVwbGFjZV9hbGwoZnVlbFssMV0sJ1llYXInLCcnKQojIGNvbG5hbWVzKGZ1ZWwpPC1zdHJfcmVwbGFjZV9hbGwoY29sbmFtZXMoZnVlbCksJyAnLCdfJykKIyBmb3IgKGkgaW4gYygzLDYsOSkpewojICAgZnVlbFssaV08LWFzLm51bWVyaWMoZ3N1YigiLCIsICIiLCBmdWVsWyxpXSkpCiMgICAKIyB9CiMgZnVlbF9vcmRlcmVkIDwtZnVlbFtucm93KGZ1ZWwpOjEsXQojIHdyaXRlLmNzdihmdWVsX29yZGVyZWQsZmlsZSA9ICdmdWVsLmNzdicpCmBgYAoKYGBge3IgZGVmaW5lIGZ1bmN0aW9uIHRvIGZldGNoIEpTT04gYnVsayBkYXRhc2V0IGZyb20gRUlBIHdlYnNpdGUgfQpnZXRfRUlBX2J1bGtfanNvbiA8LSBmdW5jdGlvbih1cmwpewogIAogIHJlcXVpcmUodXRpbHMpCiAgcmVxdWlyZShoZXJlKQogIHJlcXVpcmUoc3RyaW5ncikKICByZXF1aXJlKHppcCkKICByZXF1aXJlKGZzKQogIAogIG9zIDwtIFN5cy5pbmZvKClbWydzeXNuYW1lJ11dIAogIHNlcCA8LSBpZmVsc2Uob3MgPT0gIldpbmRvd3MiLCAiXFwiLCAiLyIpCiAgCiAgemlwX2ZpbGVuYW1lIDwtIHdvcmQodXJsLCBzdGFydD0tMSwgZW5kPS0xLCBzZXA9c2VwKQogIAogIGRvd25sb2FkX2RpciA8LSBoZXJlKCJkYXRhIiwgInJhd19kYXRhIiwgIkVJQSIpCiAgcGF0aDJ6aXAgICAgIDwtIHBhc3RlKGRvd25sb2FkX2RpciwgemlwX2ZpbGVuYW1lLCBzZXA9c2VwKQogIAogIGRvd25sb2FkLmZpbGUodXJsID0gdXJsLCBkZXN0ZmlsZSA9IHBhdGgyemlwLCBtZXRob2QgPSAibGliY3VybCIpCiAgCiAgdW56aXAocGF0aDJ6aXAsIGV4ZGlyID0gZG93bmxvYWRfZGlyKQogIAogIHVuemlwcGVkX2ZpbGVuYW1lIDwtIHppcF9saXN0KHBhdGgyemlwKSRmaWxlbmFtZQogIHBhdGgydW56aXAgPC0gcGFzdGUoZG93bmxvYWRfZGlyLHVuemlwcGVkX2ZpbGVuYW1lLHNlcD1zZXApCiAganNvbl9maWxlbmFtZSA8LSBzdHJfcmVwbGFjZSh1bnppcHBlZF9maWxlbmFtZSwgIi50eHQiLCAiLmpzb24iKQogIHBhdGgyanNvbiA8LSBwYXN0ZShkb3dubG9hZF9kaXIsanNvbl9maWxlbmFtZSxzZXA9c2VwKQogIAogIGZpbGVfbW92ZShwYXRoMnVuemlwLCBwYXRoMmpzb24pCiAgCiAgZmlsZV9kZWxldGUocGF0aDJ6aXApIAogIAogIHJldHVybihwYXRoMmpzb24pCn0KYGBgCgpgYGB7ciBmZXRjaCBhbmQgcGFyc2UgU0VEUyBidWxrIEpTT04gZmlsZSBmcm9tIEVJQSB3ZWJzaXRlfQpsaWJyYXJ5KGpzb25saXRlKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KGZzdCkKCnVybCA8LSAiaHR0cDovL2FwaS5laWEuZ292L2J1bGsvU0VEUy56aXAiCgpwYXRoMmpzb24gPC0gZ2V0X0VJQV9idWxrX2pzb24odXJsKQoKU0VEUyA8LSBzdHJlYW1faW4oZmlsZShwYXRoMmpzb24pKQoKKFNFRFMgJT4lCiAgYXNfdGliYmxlICU+JQogIGZpbHRlcihnZW9ncmFwaHkgPT0gIlVTQS1WQSIgJiBzdHJfZGV0ZWN0KGRlc2NyaXB0aW9uLCJsZWN0cmljaXR5IikpICU+JQogIGZsYXR0ZW4oKSAlPiUKICBtdXRhdGVfYWxsKGFzLmNoYXJhY3RlciktPiBTRURTX3ZhX2VsZWMpCgp3cml0ZV9mc3QoU0VEU192YV9lbGVjLCBwYXRoID0gaGVyZSgiZGF0YSIsICJTRURTX3ZhX2VsZWMuZnN0IikpCgpgYGAKCmBgYHtyIGZldGNoIGFuZCBwYXJzZSBFTEVDIGJ1bGsgSlNPTiBmaWxlIGZyb20gRUlBIHdlYnNpdGV9CmxpYnJhcnkoanNvbmxpdGUpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkoZnN0KQoKdXJsIDwtICJodHRwOi8vYXBpLmVpYS5nb3YvYnVsay9FTEVDLnppcCIKCnBhdGgyanNvbiA8LSBnZXRfRUlBX2J1bGtfanNvbih1cmwpCgpFTEVDIDwtIHN0cmVhbV9pbihmaWxlKHBhdGgyanNvbikpCgooRUxFQyAlPiUKICBhc190aWJibGUgJT4lCiAgZmlsdGVyKGdlb2dyYXBoeSA9PSAiVVNBLVZBIikgJT4lCiAgZmxhdHRlbigpICU+JQogIG11dGF0ZV9hbGwoYXMuY2hhcmFjdGVyKSAtPiBFTEVDX3ZhKQoKd3JpdGVfZnN0KEVMRUNfdmEsIHBhdGggPSBoZXJlKCJkYXRhIiwgIkVMRUNfdmEuZnN0IikpCgpgYGAKCmBgYHtyIGZldGNoIGFuZCBwYXJzZSBidWxrIEpTT04gZmlsZSBvbiBVUyBFbGVjdHJpYyBTeXN0ZW0gT3BlcmF0aW5nIERhdGEgZnJvbSBFSUEgd2Vic2l0ZX0KdXJsIDwtICJodHRwOi8vYXBpLmVpYS5nb3YvYnVsay9FQkEuemlwIgoKcGF0aDJqc29uIDwtIGdldF9FSUFfYnVsa19qc29uKHVybCkKCkVCQSA8LSBzdHJlYW1faW4oZmlsZShwYXRoMmpzb24pKQoKKEVCQSAlPiUKICBhc190aWJibGUgJT4lCiAgZmlsdGVyKGdlb2dyYXBoeSA9PSAiVVNBLVZBIikgJT4lCiAgZmxhdHRlbigpICU+JQogIG11dGF0ZV9hbGwoYXMuY2hhcmFjdGVyKSAtPiBFQkFfdmEpCgp3cml0ZV9mc3QoRUJBX3ZhLCBwYXRoID0gaGVyZSgiZGF0YSIsICJFQkFfdmEuZnN0IikpCgpgYGAKYGBge3IgcmVhZCBpbiBTRURTX3ZhX2VsZWN9CihTRURTX3ZhX2VsZWMgPC0gcmVhZF9mc3QocGF0aCA9IGhlcmUoImRhdGEiLCAiU0VEU192YV9lbGVjLmZzdCIpKSkKYGBgCgpgYGB7ciByZWFkIGluIEVMRUNfdmF9CihFTEVDX3ZhIDwtIHJlYWRfZnN0KHBhdGggPSBoZXJlKCJkYXRhIiwgIkVMRUNfdmEuZnN0IikpKQpgYGA=